import os
import re
import sys
import time
import traceback
from datetime import datetime

import logzero
import requests
from logzero import logger
from lxml import html
from tqdm import tqdm
from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException

from config import REPORT_ERROR_COLLECTION_NAME
from utils import open_driver, track_alert, get_mongo_collection, record_run_status

COLLECTION_NAME = 'bcy_cos_list'
COLLECTION_FEED = 'bcy_cos_detail_feed'
current_module = os.path.splitext(os.path.basename(__file__))[0]
logzero.logfile(f"/tmp/{current_module}.log", maxBytes=30_000_000, backupCount=3, encoding='utf-8')

db = get_mongo_collection(COLLECTION_NAME)
db_feed = get_mongo_collection(COLLECTION_FEED)
error_db = get_mongo_collection(REPORT_ERROR_COLLECTION_NAME)
RE_ENDPAGE = re.compile(r'.*p=(\d+)')

insert_count = 0


@record_run_status(current_module)
def spider(page_begin=1):
    stop = False
    # 索引页是半次元cos热门cos，最新正片
    index_url = 'https://bcy.net/coser/allwork?&p={p}'
    url = index_url.format(p=page_begin)
    total_page = get_total_page(url)
    with open_driver() as driver:
        with track_alert(driver):
            for p in tqdm(range(page_begin, total_page)):
                if stop:
                    break
                logger.debug('*' * 50)
                try:
                    url = index_url.format(p=p)
                    image_cards_ele = get_image_cards_ele(driver, url)
                    print(len(image_cards_ele))
                    for image_card_ele in image_cards_ele:
                        stop = handle_data(image_card_ele)
                        if stop:
                            break
                except (NoSuchElementException, StaleElementReferenceException) as e:
                    handle_exception(e, p)
                else:
                    delete_error(p)
            logger.info('完成')
            return insert_count


def get_total_page(url):
    resp = requests.get(url).text
    root = html.fromstring(resp)
    end_page_href = root.cssselect('ul.pager>li:last-child>a')[0].attrib['href']
    total_page = int(RE_ENDPAGE.search(end_page_href).group(1))
    logger.info(f'总共{total_page}页')
    return total_page


def get_image_cards_ele(driver, url):
    driver.get(url)
    logger.info(f'访问{url}')
    time.sleep(3)
    image_cards_ele = driver.find_elements_by_css_selector('ul.gridList>li')
    return image_cards_ele


def handle_data(image_card_ele):
    """
    处理每条数据
    :param image_card_ele:
    :return: 是否停止
    """
    img = image_card_ele.find_element_by_css_selector('img').get_attribute(
        'src')
    a = image_card_ele.find_element_by_css_selector('a')
    href = a.get_attribute('href')
    title = a.get_attribute('title')
    worker = image_card_ele.find_element_by_css_selector(
        ' a.name > span').text
    character = image_card_ele.find_element_by_css_selector(
        ' div > p > span').text
    data = {'img': img, 'href': href, 'title': title, 'worker': worker, 'character': character,
            'update_time': datetime.now()}
    return store_data(data, href)


def store_data(data, href):
    global insert_count
    stop = False
    old_record = db.find_one({'href': href})
    if old_record:
        logger.info(f'{href} 已经存储，停止爬取')
        # stop = True
    else:
        logger.info(f'{href}不存在，现在插入')
        logger.info(data)
        db.insert_one(data)
        db_feed.insert_one({'href': href, 'update_time': datetime.now()})
        insert_count += 1
    print(stop)
    return stop


def handle_exception(e, p):
    """
    处理异常
    :param e: 异常
    :param p: 页码
    :return:
    """
    logger.error(e)
    logger.error(traceback.format_exc())
    error_record_condition = {'desp': COLLECTION_NAME, 'page': p}
    error_record = error_db.find_one(error_record_condition)
    if not error_record:
        error_db.insert_one(
            {'desp': COLLECTION_NAME, 'page': p, 'e': e,
             'e_traceback': traceback.format_exc(), 'update_time': datetime.now()})


def delete_error(p):
    error_record_condition = {'desp': COLLECTION_NAME, 'page': p}
    error_record = error_db.find_one(error_record_condition)
    if error_record:
        error_db.delete_one(error_record_condition)
        logger.info('删掉异常记录')


if __name__ == '__main__':
    """
    如果从某页码开始访问在后面直接加参数
    """
    page_begin = int(sys.argv[1]) if len(sys.argv) == 2 else 1
    spider(page_begin)
